#!/usr/bin/perl
use warnings;
use strict;
use Getopt::Long;
use LWP;
use LWP::UserAgent;
use Storable;
use File::Path;
use threads;
use Thread::Queue;

die "Usage: $0 <urls_file> <dump_dir> [thread_cnt]\n" unless @ARGV >= 2;
my $urls_file = shift;
my $dump_dir = shift;
my $thread_cnt = shift || 50;

mkdir $dump_dir unless -d $dump_dir;

my $url_queue = Thread::Queue->new;

my $downloadCount : shared = 0;

open IN, "< $urls_file" or die "$!";
chdir $dump_dir or die "$!";
while (<IN>) {
  chomp;
  $url_queue->enqueue($_);
}
print $url_queue->pending, " seed URLs read in!\n";
close IN;

chdir $dump_dir;

my @threads = ();

for (1..$thread_cnt) {
  push @threads, 	threads->new( sub {
					my $ua = LWP::UserAgent->new;
					$ua->timeout(120);
					$ua->agent('Mozilla/8.0');
					$ua->max_size(128000);
					while (my $url = $url_queue->dequeue_nb) {
					    (my $file = $url) =~ s/\//\^/g;
					  my $start_time = time;
					  my $response = $ua->get($url);
					  my $elapsed_time = time - $start_time;
					  if ($response->is_success) {
					    my $content_type = $response->header('Content-Type');
					    if ($content_type and $content_type !~  m{^text/html|^\s*$}i) {
					      next;
					    }
					    my $content_lang = $response->header('Content-Language');
					    if ($content_lang and $content_lang !~ /^en|^\s*$/i) {
					      next;
					    }
					    my $content = $response->content;
					    next unless $content;
					    open OUT, "> $file" or die "$!";
					    print OUT $content;
					    close OUT;
					    {
					      lock ($downloadCount);
					      $downloadCount++;
					      print "[No.$downloadCount] $url ($elapsed_time sec.) [Thread ", threads->self->tid, "]\n";
					    }
					  }
					}
					print "Thread No. ", threads->self->tid, " finished!\n";
				      });
}
for (@threads) {
  $_->join;
}
print "All threads ended!\n";
print "Finished!\n";
